{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#|default_exp models.TSSequencerPlus"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# TSSequencerPlus"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    ">This is a PyTorch implementation created by Ignacio Oguiza (oguiza@timeseriesAI.co) based on Sequencer: Deep LSTM for Image Classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#|export\n",
    "from tsai.imports import *\n",
    "from tsai.models.utils import *\n",
    "from tsai.models.layers import *\n",
    "from typing import Callable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#|export\n",
    "class _TSSequencerEncoderLayer(nn.Module):\n",
    "    def __init__(self, d_model:int, q_len:int=None, lstm_dropout:float=0., dropout:float=0, drop_path_rate:float=0.,\n",
    "                 mlp_ratio:int=1, lstm_bias:bool=True, act:str='gelu', pre_norm:bool=False):\n",
    "        super().__init__()\n",
    "        self.bilstm = nn.LSTM(q_len, q_len, num_layers=1, bidirectional=True, bias=lstm_bias)\n",
    "        self.dropout = nn.Dropout(lstm_dropout) if lstm_dropout else nn.Identity()\n",
    "        self.fc = nn.Linear(2 * q_len, q_len)\n",
    "        self.lstm_norm = nn.LayerNorm(d_model)\n",
    "        self.pwff =  PositionwiseFeedForward(d_model, dropout=dropout, act=act, mlp_ratio=mlp_ratio)\n",
    "        self.ff_norm = nn.LayerNorm(d_model)\n",
    "        self.drop_path = DropPath(drop_path_rate) if drop_path_rate != 0 else nn.Identity()\n",
    "        self.pre_norm = pre_norm\n",
    "        self.transpose = Transpose(1,2)\n",
    "\n",
    "    def forward(self, x):\n",
    "        if self.pre_norm:\n",
    "            x = self.drop_path(self.dropout(self.transpose(self.fc(self.bilstm(self.transpose(self.lstm_norm(x)))[0])))) + x\n",
    "            x = self.drop_path(self.pwff(self.ff_norm(x))) + x\n",
    "        else:\n",
    "            x = self.lstm_norm(self.drop_path(self.dropout(self.transpose(self.fc(self.bilstm(self.transpose(x))[0])))) + x)\n",
    "            x = self.ff_norm(self.drop_path(self.pwff(x)) + x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#|export\n",
    "class _TSSequencerEncoder(nn.Module):\n",
    "    def __init__(self, d_model, depth:int=6, q_len:int=None, lstm_dropout:float=0., dropout:float=0, drop_path_rate:float=0.,\n",
    "                 mlp_ratio:int=1, lstm_bias:bool=True, act:str='gelu', pre_norm:bool=False):\n",
    "        super().__init__()\n",
    "        dpr = [x.item() for x in torch.linspace(0, drop_path_rate, depth)]\n",
    "        layers = []\n",
    "        for i in range(depth):\n",
    "            layer = _TSSequencerEncoderLayer(d_model, q_len=q_len, lstm_dropout=lstm_dropout, dropout=dropout, drop_path_rate=dpr[i],\n",
    "                                      mlp_ratio=mlp_ratio, lstm_bias=lstm_bias, act=act, pre_norm=pre_norm)\n",
    "            layers.append(layer)\n",
    "        self.encoder = nn.Sequential(*layers)\n",
    "        self.norm = nn.LayerNorm(d_model) if pre_norm else nn.Identity()\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.encoder(x)\n",
    "        x = self.norm(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#|export\n",
    "class _TSSequencerBackbone(Module):\n",
    "    def __init__(self, c_in:int, seq_len:int, depth:int=6, d_model:int=128, act:str='gelu',\n",
    "                 lstm_bias:bool=True, lstm_dropout:float=0., dropout:float=0., drop_path_rate:float=0., mlp_ratio:int=1,\n",
    "                 pre_norm:bool=False, use_token:bool=True,  use_pe:bool=True, n_cat_embeds:Optional[list]=None, cat_embed_dims:Optional[list]=None,\n",
    "                 cat_padding_idxs:Optional[list]=None, cat_pos:Optional[list]=None, feature_extractor:Optional[Callable]=None,\n",
    "                 token_size:int=None, tokenizer:Optional[Callable]=None):\n",
    "\n",
    "        # Categorical embeddings\n",
    "        if n_cat_embeds is not None:\n",
    "            n_cat_embeds = listify(n_cat_embeds)\n",
    "            if cat_embed_dims is None:\n",
    "                cat_embed_dims = [emb_sz_rule(s) for s in n_cat_embeds]\n",
    "            self.to_cat_embed = MultiEmbedding(c_in, n_cat_embeds, cat_embed_dims=cat_embed_dims, cat_padding_idxs=cat_padding_idxs, cat_pos=cat_pos)\n",
    "            c_in, seq_len = output_size_calculator(self.to_cat_embed, c_in, seq_len)\n",
    "        else:\n",
    "            self.to_cat_embed = nn.Identity()\n",
    "\n",
    "        # Sequence embedding\n",
    "        if token_size is not None:\n",
    "            self.tokenizer = SeqTokenizer(c_in, d_model, token_size)\n",
    "            c_in, seq_len = output_size_calculator(self.tokenizer, c_in, seq_len)\n",
    "        elif tokenizer is not None:\n",
    "            if isinstance(tokenizer, nn.Module):  self.tokenizer = tokenizer\n",
    "            else: self.tokenizer = tokenizer(c_in, d_model)\n",
    "            c_in, seq_len = output_size_calculator(self.tokenizer, c_in, seq_len)\n",
    "        else:\n",
    "            self.tokenizer = nn.Identity()\n",
    "\n",
    "        # Feature extractor\n",
    "        if feature_extractor is not None:\n",
    "            if isinstance(feature_extractor, nn.Module):  self.feature_extractor = feature_extractor\n",
    "            else: self.feature_extractor = feature_extractor(c_in, d_model)\n",
    "            c_in, seq_len = output_size_calculator(self.feature_extractor, c_in, seq_len)\n",
    "        else:\n",
    "            self.feature_extractor = nn.Identity()\n",
    "\n",
    "        # Linear projection\n",
    "        self.transpose = Transpose(1,2)\n",
    "        if token_size is None and tokenizer is None and feature_extractor is None:\n",
    "            self.linear_proj = nn.Linear(c_in, d_model)\n",
    "            # self.linear_proj = nn.Conv1d(c_in, d_model, 1)\n",
    "        else:\n",
    "            self.linear_proj = nn.Identity()\n",
    "\n",
    "        # Position embedding & token\n",
    "        if use_pe:\n",
    "            self.pos_embed = nn.Parameter(torch.zeros(1, seq_len, d_model))\n",
    "        self.use_pe = use_pe\n",
    "        self.cls_token = nn.Parameter(torch.zeros(1, 1, d_model))\n",
    "        self.use_token = use_token\n",
    "        self.emb_dropout = nn.Dropout(dropout) if dropout else nn.Identity()\n",
    "\n",
    "        # Encoder\n",
    "        self.encoder = _TSSequencerEncoder(d_model, depth=depth, q_len=seq_len + use_token, lstm_bias=lstm_bias,\n",
    "                                         lstm_dropout=lstm_dropout, dropout=dropout,\n",
    "                                         mlp_ratio=mlp_ratio, drop_path_rate=drop_path_rate, act=act, pre_norm=pre_norm)\n",
    "\n",
    "    def forward(self, x):\n",
    "\n",
    "        # Categorical embeddings\n",
    "        x = self.to_cat_embed(x)\n",
    "\n",
    "        # Sequence embedding\n",
    "        x = self.tokenizer(x)\n",
    "\n",
    "        # Feature extractor\n",
    "        x = self.feature_extractor(x)\n",
    "\n",
    "        # Linear projection\n",
    "        x = self.transpose(x)\n",
    "        x = self.linear_proj(x)\n",
    "\n",
    "        # Position embedding & token\n",
    "        if self.use_pe:\n",
    "            x = x + self.pos_embed\n",
    "        if self.use_token: # token is concatenated after position embedding so that embedding can be learned using self.supervised learning\n",
    "            x = torch.cat((self.cls_token.expand(x.shape[0], -1, -1), x), dim=1)\n",
    "        x = self.emb_dropout(x)\n",
    "\n",
    "        # Encoder\n",
    "        x = self.encoder(x)\n",
    "\n",
    "        # Output\n",
    "        x = x.transpose(1,2)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#|exports\n",
    "class TSSequencerPlus(nn.Sequential):\n",
    "    r\"\"\"Time Series Sequencer model based on:\n",
    "\n",
    "    Tatsunami, Y., & Taki, M. (2022). Sequencer: Deep LSTM for Image Classification. arXiv preprint arXiv:2205.01972.\n",
    "    Official implementation: https://github.com/okojoalg/sequencer\n",
    "\n",
    "    Args:\n",
    "        c_in:               the number of features (aka variables, dimensions, channels) in the time series dataset.\n",
    "        c_out:              the number of target classes.\n",
    "        seq_len:            number of time steps in the time series.\n",
    "        d_model:            total dimension of the model (number of features created by the model).\n",
    "        depth:              number of blocks in the encoder.\n",
    "        act:                the activation function of positionwise feedforward layer.\n",
    "        lstm_dropout:       dropout rate applied to the lstm sublayer.\n",
    "        dropout:            dropout applied to to the embedded sequence steps after position embeddings have been added and\n",
    "                            to the mlp sublayer in the encoder.\n",
    "        drop_path_rate:     stochastic depth rate.\n",
    "        mlp_ratio:          ratio of mlp hidden dim to embedding dim.\n",
    "        lstm_bias:          determines whether bias is applied to the LSTM layer.\n",
    "        pre_norm:           if True normalization will be applied as the first step in the sublayers. Defaults to False.\n",
    "        use_token:          if True, the output will come from the transformed token. This is meant to be use in classification tasks.\n",
    "        use_pe:             flag to indicate if positional embedding is used.\n",
    "        n_cat_embeds:       list with the sizes of the dictionaries of embeddings (int).\n",
    "        cat_embed_dims:     list with the sizes of each embedding vector (int).\n",
    "        cat_padding_idxs:       If specified, the entries at cat_padding_idxs do not contribute to the gradient; therefore, the embedding vector at cat_padding_idxs\n",
    "                            are not updated during training. Use 0 for those categorical embeddings that may have #na# values. Otherwise, leave them as None.\n",
    "                            You can enter a combination for different embeddings (for example, [0, None, None]).\n",
    "        cat_pos:            list with the position of the categorical variables in the input.\n",
    "        token_size:         Size of the embedding function used to reduce the sequence length (similar to ViT's patch size)\n",
    "        tokenizer:          nn.Module or callable that will be used to reduce the sequence length\n",
    "        feature_extractor:  nn.Module or callable that will be used to preprocess the time series before\n",
    "                            the embedding step. It is useful to extract features or resample the time series.\n",
    "        flatten:            flag to indicate if the 3d logits will be flattened to 2d in the model's head if use_token is set to False.\n",
    "                            If use_token is False and flatten is False, the model will apply a pooling layer.\n",
    "        concat_pool:        if True the head begins with fastai's AdaptiveConcatPool2d if concat_pool=True; otherwise, it uses traditional average pooling.\n",
    "        fc_dropout:         dropout applied to the final fully connected layer.\n",
    "        use_bn:             flag that indicates if batchnorm will be applied to the head.\n",
    "        bias_init:          values used to initialized the output layer.\n",
    "        y_range:            range of possible y values (used in regression tasks).\n",
    "        custom_head:        custom head that will be applied to the network. It must contain all kwargs (pass a partial function)\n",
    "        verbose:            flag to control verbosity of the model.\n",
    "\n",
    "    Input:\n",
    "        x: bs (batch size) x nvars (aka features, variables, dimensions, channels) x seq_len (aka time steps)\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, c_in:int, c_out:int, seq_len:int, d_model:int=128, depth:int=6, act:str='gelu',\n",
    "                 lstm_dropout:float=0., dropout:float=0., drop_path_rate:float=0., mlp_ratio:int=1, lstm_bias:bool=True,\n",
    "                 pre_norm:bool=False, use_token:bool=False, use_pe:bool=True,\n",
    "                 cat_pos:Optional[list]=None, n_cat_embeds:Optional[list]=None, cat_embed_dims:Optional[list]=None, cat_padding_idxs:Optional[list]=None,\n",
    "                 token_size:int=None, tokenizer:Optional[Callable]=None, feature_extractor:Optional[Callable]=None,\n",
    "                 flatten:bool=False, concat_pool:bool=True, fc_dropout:float=0., use_bn:bool=False,\n",
    "                 bias_init:Optional[Union[float, list]]=None, y_range:Optional[tuple]=None, custom_head:Optional[Callable]=None, verbose:bool=True,\n",
    "                 **kwargs):\n",
    "\n",
    "        if use_token and c_out == 1:\n",
    "            use_token = False\n",
    "            pv(\"use_token set to False as c_out == 1\", verbose)\n",
    "        backbone = _TSSequencerBackbone(c_in, seq_len, depth=depth, d_model=d_model, act=act,\n",
    "                                      lstm_dropout=lstm_dropout, dropout=dropout, drop_path_rate=drop_path_rate,\n",
    "                                      pre_norm=pre_norm, mlp_ratio=mlp_ratio, use_pe=use_pe, use_token=use_token,\n",
    "                                      n_cat_embeds=n_cat_embeds, cat_embed_dims=cat_embed_dims, cat_padding_idxs=cat_padding_idxs, cat_pos=cat_pos,\n",
    "                                      feature_extractor=feature_extractor, token_size=token_size, tokenizer=tokenizer)\n",
    "\n",
    "        self.head_nf = d_model\n",
    "        self.c_out = c_out\n",
    "        self.seq_len = seq_len\n",
    "\n",
    "        # Head\n",
    "        if custom_head:\n",
    "            if isinstance(custom_head, nn.Module): head = custom_head\n",
    "            else: head = custom_head(self.head_nf, c_out, seq_len, **kwargs)\n",
    "        else:\n",
    "            nf = d_model\n",
    "            layers = []\n",
    "            if use_token:\n",
    "                layers += [TokenLayer()]\n",
    "            elif flatten:\n",
    "                layers += [Reshape(-1)]\n",
    "                nf = nf * seq_len\n",
    "            else:\n",
    "                if concat_pool: nf *= 2\n",
    "                layers = [GACP1d(1) if concat_pool else GAP1d(1)]\n",
    "            if use_bn: layers += [nn.BatchNorm1d(nf)]\n",
    "            if fc_dropout: layers += [nn.Dropout(fc_dropout)]\n",
    "\n",
    "            # Last layer\n",
    "            linear = nn.Linear(nf, c_out)\n",
    "            if bias_init is not None:\n",
    "                if isinstance(bias_init, float): nn.init.constant_(linear.bias, bias_init)\n",
    "                else: linear.bias = nn.Parameter(torch.as_tensor(bias_init, dtype=torch.float32))\n",
    "            layers += [linear]\n",
    "\n",
    "            if y_range: layers += [SigmoidRange(*y_range)]\n",
    "            head = nn.Sequential(*layers)\n",
    "        super().__init__(OrderedDict([('backbone', backbone), ('head', head)]))\n",
    "\n",
    "\n",
    "TSSequencer = TSSequencerPlus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bs = 16\n",
    "nvars = 4\n",
    "seq_len = 50\n",
    "c_out = 2\n",
    "xb = torch.rand(bs, nvars, seq_len)\n",
    "model = TSSequencerPlus(nvars, c_out, seq_len)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bs = 16\n",
    "nvars = 4\n",
    "seq_len = 50\n",
    "c_out = 2\n",
    "xb = torch.rand(bs, nvars, seq_len)\n",
    "model = TSSequencerPlus(nvars, c_out, seq_len, lstm_dropout=.1, dropout=.1, use_token=True)\n",
    "test_eq(model(xb).shape, (bs, c_out))\n",
    "model = TSSequencerPlus(nvars, c_out, seq_len, lstm_dropout=.1, dropout=.1, use_token=False)\n",
    "test_eq(model(xb).shape, (bs, c_out))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bs = 16\n",
    "nvars = 4\n",
    "seq_len = 50\n",
    "c_out = 2\n",
    "xb = torch.rand(bs, nvars, seq_len)\n",
    "bias_init = np.array([0.8, .2])\n",
    "model = TSSequencerPlus(nvars, c_out, seq_len, bias_init=bias_init)\n",
    "test_eq(model(xb).shape, (bs, c_out))\n",
    "test_eq(model.head[1].bias.data, tensor(bias_init))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bs = 16\n",
    "nvars = 4\n",
    "seq_len = 50\n",
    "c_out = 1\n",
    "xb = torch.rand(bs, nvars, seq_len)\n",
    "bias_init = 8.5\n",
    "model = TSSequencerPlus(nvars, c_out, seq_len, bias_init=bias_init)\n",
    "test_eq(model(xb).shape, (bs, c_out))\n",
    "test_eq(model.head[1].bias.data, tensor([bias_init]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bs = 16\n",
    "nvars = 4\n",
    "seq_len = 50\n",
    "c_out = 2\n",
    "xb = torch.rand(bs, nvars, seq_len)\n",
    "bias_init = np.array([0.8, .2])\n",
    "model = TSSequencerPlus(nvars, c_out, seq_len, bias_init=bias_init)\n",
    "test_eq(model(xb).shape, (bs, c_out))\n",
    "test_eq(model.head[1].bias.data, tensor(bias_init))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Feature extractor\n",
    "\n",
    "It's a known fact that transformers cannot be directly applied to long sequences. To avoid this, we have included a way to subsample the sequence to generate a more manageable input."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tsai.data.validation import get_splits\n",
    "from tsai.data.core import get_ts_dls"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAABZcAAABoCAYAAACNDM73AAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjAsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvlHJYcgAAAAlwSFlzAAAPYQAAD2EBqD+naQAAGAdJREFUeJzt3QlUlPX3x/HLpiib4oKiKGZY5ppLppnpKc0sS8rMslIrWyypSCs6BWomleUxF0qzpFNatmm0WKnl2qLmUqaSu1SWlgpCKYrzP/f7b/gNOgIPMDPM8H6dM2eYYeZ57jz4CHy4c79+NpvNJgAAAAAAAAAAWOBv5cEAAAAAAAAAACjCZQAAAAAAAACAZYTLAAAAAAAAAADLCJcBAAAAAAAAAJYRLgMAAAAAAAAALCNcBgAAAAAAAABYRrgMAAAAAAAAALCMcBkAAAAAAAAAYBnhMgAAAAAAAADAMsJlAAAAF0lPTxc/Pz/Zs2dP4X09e/Y0l4o2duxYsy9HsbGxMmzYMHE1fX26b329drrf0NBQcRfdvx4DAAAAAO5DuAwAAPCfn376SQYOHChNmzaV4OBgadSokfTu3VumTZvmsn3+/vvvJhTduHGjVAafffZZpQ1pK3NtAAAAQFUU6OkCAAAAKoNvvvlGevXqJU2aNJERI0ZIgwYNJCsrS7777jt56aWXZNSoURWyny+//PKMcHncuHGmy7h9+/ZSkTIzM8Xf399ygDtjxgxLIa6G8f/++68EBQWVocqKqU33HxjIj7YAAACAO/ETOAAAgIg888wzEhERIWvXrpVatWoV+dyBAwcqbD/VqlUTd6levbpLt3/y5Ek5deqUeU3a6e1Jnt4/AAAAUBUxFgMAAEBEdu7cKa1atTojWFb169c/Y77vAw88IHPnzpXzzjvPBJsdO3aUFStWlLgfx5nLy5Ytk86dO5uPhw8fbrZ7+uxiZ1atWmWep/tt3ry5zJw50+njTp+5fOLECdMlHRcXZ55bp04d6d69uyxevNh8Xh+rncH212i/OM5VfuGFF2TKlClmvxpeb9myxenMZbtdu3bJlVdeKSEhIRIdHS3jx48Xm81W+Hk9BvpcvXZ0+jaLq81+3+kdzRs2bJCrrrpKwsPDzfznyy+/3HSiO5uLvXr1aklMTJR69eqZWuPj4+XgwYPFfh0AAACAqo7OZQAAgP9GO3z77beyefNmad26dYmPX758ucyfP18SEhJMyJqWliZ9+/aVNWvWlOr5qmXLliZsTU5OlrvvvlsuvfRSc3+3bt2KnQvdp08fE4JqmKrdwykpKRIVFVXi/vTxqampctddd8lFF10kOTk5sm7dOlm/fr2ZLX3PPfeYMR0aNr/55ptOtzFnzhw5duyYqVdfd2RkpOledqagoMAck4svvlief/55+fzzz02tWrO+bitKU5ujn3/+2RxPDZYfffRRM7JDQ3gN9vVr16VLlyKP17EntWvXNvVpsK0Buv4BQb/GAAAAAJwjXAYAABCR0aNHmy5XnXuswasGk9rpqnOYnc0S1hBag1ntWFaDBw82XcwaFH/44Yel2qcGwrpPfU7Xrl3l1ltvLfE5+ljt/F25cqWZD61uuOEGadOmTYnP/fTTT6Vfv34ya9Ysp5/XGlq0aGEC3LPV8uuvv8qOHTtMuG2nYawzGkJruDx16lRze+TIkdK/f3957rnnTChft27dEmu2UpujJ5980nRqa5f3OeecY+67/fbbzddIw2YNmB1pF7fOw7Z3Q2tgrnVnZ2ebcSkAAAAAzsRYDAAAABHTuaudy9dee61s2rTJdNrqOIdGjRpJRkaG07DTHiwrDXqvu+46+eKLL0zHrivodnX7AwYMKAyW7R3QWmtJdOSHdvRu3769zDVokO0YLJdEu39PHyeSn58vS5YsEVfR46RBsR4ne7CsGjZsKLfccosJnLVr25F2YjuO2dA/Luh29u7d67I6AQAAAG9HuAwAAPAfnWOsXceHDx824y2SkpLk6NGjMnDgQDNb2JHOLT6ddtb+888/LpvVq9v9999/ne5bO3JLoqMojhw5YurUTucxY8bIjz/+aKmGZs2alfqx/v7+RcJdpfsurtu5oo6Tfh2cHRMN4rUrOSsrq8j9jmG90hEZSv8tAAAAAHCOcBkAAOA01apVM0HzxIkT5eWXXzbjFd577z3xdj169DALF77++utmLvTs2bOlQ4cO5rq0atSoUaE1OXYLO3JV9/fZBAQEOL3fcfFBAAAAAEURLgMAABSjU6dO5nr//v1F7nc2WuKXX36RmjVrWhobcbZw1Rndroa7zvadmZlZqm3oAnzDhw+Xt99+23Tvtm3b1iz0V5Z6SqIdwrt27TrjGKnY2NgiHcLaUe3I2TiK0tamx0m/Ds6OybZt20xHdUxMjIVXAgAAAMAZwmUAAAAR+frrr512qX722Wfm+vQRCzqfef369YW3Naj96KOPpE+fPmftgnUmJCTEabjqjG5XZysvXLhQ9u3bV3j/1q1bzSzmkvz9999FboeGhsq5554rx48fL1M9pTF9+vTCj/X46m1dIFEXS1RNmzY1r2vFihVFnpeWlnbGtkpbm25Pvw769XAcv/Hnn3/KvHnzpHv37hIeHl7u1wYAAABUdYGeLgAAAKAyGDVqlJnTGx8fL+eff75ZdO6bb76R+fPnmy5b7fZ1pGMlNOhNSEiQ6tWrF4ah48aNs7Tf5s2bm4X2XnnlFQkLCzMBapcuXc4621i3//nnn5sF50aOHCknT56UadOmSatWrUqcn3zBBRdIz549zUKE2sG8bt06ef/994ssumdfpFBfl74+DWoHDx4sZREcHGxqHTp0qHlNixYtkk8//VSeeOKJwu7uiIgIufHGG81r0M5kPR6ffPKJHDhw4IztWaltwoQJsnjxYhMk63EKDAyUmTNnmiBdF2sEAAAAUH6EywAAACLywgsvmLnK2qk8a9YsEy7rIm8aTD755JMmAHZ02WWXSdeuXU3Yq13EGtymp6ebMRNWaBfvG2+8YRYPvPfee01YPGfOnLOGy7p97VJOTEyU5ORkady4salBx3aUFC5rKJuRkSFffvmlCVm1a1hDWF3Yz+766683Qfs777wjb731luk2Lmu4rOGvhsv33Xef2YeG5ykpKaZuRxos61xrDdg1qB80aJBMmjTJBPiOrNSmYfvKlSvNcU1NTTUjOjTg1ufpNQAAAIDy87OxSgkAAIAl2mF7//33Fxn5AAAAAABVDTOXAQAAAAAAAACWES4DAAAAAAAAACwjXAYAAAAAAAAAWMaCfgAAABaxZAUAAAAA0LkMAAAAAAAAACgDwmUAAAAAAAAAQOUfi3Hq1Cn5/fffJSwsTPz8/Ny9ewAAAAAAAMDrx7QdPXpUoqOjxd+f3lFUoXBZg+WYmBh37xYAAAAAAADwKVlZWdK4cWNPl4EqzO3hsnYs/78sEQl39+4BAAAAAKgy2i3v4ekSALhAQV6BbO632SFnA6pIuPy/URgaLBMuAwAAAADgKgGhAZ4uAYALMXIWnsZQFgAAAAAAAACAZYTLAAAAAAAAAADLCJcBAAAAAAAAAJV/5jIAAAAAAAAAuEJBQYGcOHHC02V4rYCAAAkMDCz1PG/CZQAAAAAAAABeLzc3V3799Vex2WyeLsWr1axZUxo2bCjVqlUr8bGEywAAAAAAAAC8vmNZg2UNRuvVq1fqzlv8j4by+fn5cvDgQdm9e7fExcWJv3/xU5UJlwEAAAAAAAB4NR2FoeGoBss1atTwdDleS49dUFCQ7N271wTNwcHBxT6eBf0AAAAAAAAA+AQ6lsuvpG7lIo+tgP0BAAAAAAAAAKoYwmUAAAAAAAAAgGWEywAAAAAAAADgI2JjY2XKlClu2RfhMgAAAAAAAACfpCOY3XmxOh+6uMvYsWOlLNauXSt33323VMpwecWKFdK/f3+Jjo42L3LhwoWuqQwAAAAAAAAAfNT+/fsLL9ppHB4eXuS+0aNHFz7WZrPJyZMnS7XdevXqSc2aNaVShst5eXnSrl07mTFjhmsqAgAAAAAAAAAf16BBg8JLRESEaeS13962bZuEhYXJokWLpGPHjlK9enVZtWqV7Ny5U6677jqJioqS0NBQ6dy5syxZsqTYsRi63dmzZ0t8fLwJnePi4iQjI8Mz4fJVV10lEyZMMMUAAAAAAAAAAFzj8ccfl2effVa2bt0qbdu2ldzcXOnXr58sXbpUNmzYIH379jVTJvbt21fsdsaNGyeDBg2SH3/80Tx/yJAhcujQoco/c/n48eOSk5NT5AIAAAAAAAAAKN748eOld+/e0rx5c4mMjDQTJe655x5p3bq16UB++umnzedK6kQeNmyY3HzzzXLuuefKxIkTTUi9Zs0aqfThcmpqqmnrtl9iYmJcvUsAAAAAAAAA8HqdOnUqcltDYZ3F3LJlS6lVq5YZjaFdzSV1LmvXs11ISIiZ73zgwIHKHy4nJSVJdnZ24SUrK8vVuwQAAAAAAAAArxcSElLktgbLCxYsMN3HK1eulI0bN0qbNm0kPz+/2O0EBQUVua1zmE+dOlXu+gLFxXTYtF4AAAAAAAAAAGW3evVqM+LCvh6edjLv2bNHPMXlncsAAAAAAAAAgPLTOcsffvih6VjetGmT3HLLLRXSgey2zmVNw3fs2FF4e/fu3ebF6EDpJk2aVHR9AAAAAAAAAFAmNpv4lMmTJ8sdd9wh3bp1k7p168pjjz0mOTk5HqvHz2azdoiXLVsmvXr1OuP+oUOHSnp6eonP1xerC/uJZItIuLVqAQAAAABAqXX4oaOnSwDgAgW5BbLpsk1mfTNdmA0ix44dM02wzZo1k+DgYE+XU2WOpeXO5Z49e4rFPBoAAAAAAAAA4GOYuQwAAAAAAAAAsIxwGQAAAAAAAABgGeEyAAAAAAAAAMAywmUAAAAAAAAAgGWEywAAAAAAAAAAywiXAQAAAAAAAACWES4DAAAAAAAAACwjXAYAAAAAAAAAWEa4DAAAAAAAAACwLND6UwAAAAAAAACg8uu4vqNb9/dDhx9K/Vg/P79iP5+SkiJjx44tUx267QULFsiAAQPElQiXAQAAAAAAAMDN9u/fX/jx/PnzJTk5WTIzMwvvCw0NlcrO7eGyzWb776Mcd+8aAAAAAIAqpSC3wNMlAHCBgryC03I2eKMGDRoUfhwREWG6jR3vmz17trz44ouye/duiY2NlYSEBBk5cqT5XH5+viQmJsoHH3wghw8flqioKLn33nslKSnJPFbFx8eb66ZNm8qePXt8I1z++++///soxt27BgAAAACgStl0macrAODqnE1DSfieuXPnmk7m6dOny4UXXigbNmyQESNGSEhIiAwdOlSmTp0qGRkZ8u6770qTJk0kKyvLXNTatWulfv36MmfOHOnbt68EBAS4rE63h8uRkZHmet++ffzjB3xMTk6OxMTEmP/MwsPDPV0OgArE+Q34Ls5vwHdxfgO+Kzs72wSK9pwNviclJcV0LV9//fXmdrNmzWTLli0yc+ZMEy5rthoXFyfdu3c3Hc/anWxXr149c12rVq0indA+ES77+/ubaw2W+eYG+CY9tzm/Ad/E+Q34Ls5vwHdxfgO+y56zwbfk5eXJzp075c477zTdynYnT54sbNYdNmyY9O7dW8477zzTnXzNNddInz593F4rC/oBAAAAAAAAQCWRm5trrl999VXp0qVLkc/ZR1x06NDBzGJetGiRLFmyRAYNGiRXXHGFvP/++26tlXAZAAAAAAAAACqJqKgoiY6Oll27dsmQIUPO+jh9V8pNN91kLgMHDjQdzIcOHTLjUoKCgqSgoMD3wuXq1aubmSF6DcC3cH4DvovzG/BdnN+A7+L8BnwX57fvGzdunCQkJJgxGBoaHz9+XNatWyeHDx+WxMREmTx5sjRs2NAs9qfjUd577z0zX1nnLKvY2FhZunSpXHLJJebfSe3atV1Sp5/NZrO5ZMsAAAAAAAAA4AbHjh0zYyJ04bvg4GDxNunp6fLQQw/JkSNHCu+bN2+eTJo0ySzkFxISIm3atDGPiY+PNyMz0tLSZPv27WZURufOnc1jNWxWH3/8sQmh9+zZI40aNTLXrjiWhMsAAAAAAAAAvJq3h8veeixZUhIAAAAAAAAAYBnhMgAAAAAAAADAMsJlAAAAAAAAAIBlhMsAAAAAAAAAgModLs+YMUNiY2PNIOguXbrImjVr3Ll7AC6QmppqViQNCwuT+vXry4ABAyQzM9PTZQFwgWeffVb8/PzM6sQAvN9vv/0mt956q9SpU0dq1KhhVh9ft26dp8sCUE4FBQXy1FNPmUWY9Nxu3ry5PP3002Kz2TxdGgCLVqxYIf3795fo6Gjzc/jChQuLfF7P6+TkZGnYsKE536+44grZvn27VHX8f+feY+i2cHn+/PmSmJgoKSkpsn79emnXrp1ceeWVcuDAAXeVAMAFli9fLvfff7989913snjxYjlx4oT06dNH8vLyPF0agAq0du1amTlzprRt29bTpQCoAIcPH5ZLLrlEgoKCZNGiRbJlyxZ58cUXpXbt2p4uDUA5Pffcc/Lyyy/L9OnTZevWreb2888/L9OmTfN0aQAs0t+rNT/TZk1n9NyeOnWqvPLKK/L9999LSEiIydqOHTsmVVFAQIC5zs/P93QpXu+ff/4x1/qzYkn8bG6K87VTWbsb9RucOnXqlMTExMioUaPk8ccfd0cJANzg4MGDpoNZQ+cePXp4uhwAFSA3N1c6dOggaWlpMmHCBGnfvr1MmTLF02UBKAf9+Xv16tWycuVKT5cCoIJdc801EhUVJa+99lrhfTfccIPpanzrrbc8WhuAstPO5QULFph3CyuN87Sj+ZFHHpHRo0eb+7Kzs835n56eLoMHD5aqRo/Jvn37TNObHht/f6YBl+UYarCszcC1atUyXfElCRQ30L8Y/PDDD5KUlFR4n36BtV3/22+/dUcJANxEv5mpyMhIT5cCoILouxOuvvpq831bw2UA3i8jI8N0Nt14443mD8KNGjWSkSNHyogRIzxdGoBy6tatm8yaNUt++eUXadGihWzatElWrVolkydP9nRpACrQ7t275Y8//jA/o9tFRESY5k7N2qpiuKwBvIahemz27t3r6XK8mgbLDRo0KNVj3RIu//XXX2buk/71xJHe3rZtmztKAOAG+o4EncWqb7Nt3bq1p8sBUAHeeecdM85Kx2IA8B27du0yb5vXsXVPPPGEOccTEhKkWrVqMnToUE+XB6Cc70zIycmR888/37xFXH8Xf+aZZ2TIkCGeLg1ABdJgWTnL2uyfq4r0Z5m4uDhGY5SDjsKwjxipNOEygKrT3bh582bTGQHA+2VlZcmDDz5o5qnrYrwAfOsPwp06dZKJEyea2xdeeKH5Hq4zGwmXAe/27rvvyty5c2XevHnSqlUr2bhxo2kA0beIc34DqAp0WgK/v7iPW4aP1K1b1yTef/75Z5H79XZpW6wBVG4PPPCAfPLJJ/L1119L48aNPV0OgAqgI6101pbOWw4MDDQXffu8LhqiH2snFADvpG8ZveCCC4rc17JlSzOnEIB3GzNmjOle1rfEt2nTRm677TZ5+OGHJTU11dOlAahA9jyNrA1VIlzWlvSOHTvK0qVLi3RL6O2uXbu6owQALhz2rsGyLizw1VdfSbNmzTxdEoAKcvnll8tPP/1kOp7sF+101LfV6sdW3ioFoHLREVaZmZlF7tP5rE2bNvVYTQAqhi7EdPoiVvo9W38HB+A79HdvDZEdszYdifP999+TtcGt3DYWQ+e56Vtw9JfSiy66yKwyn5eXJ8OHD3dXCQBcNApD33L30UcfSVhYWOFsJ11IQFekBuC99Jw+fX56SEiI1KlTh7nqgJfTLkZd9EvHYgwaNEjWrFljFgDTCwDv1r9/fzNjuUmTJmYsxoYNG8xifnfccYenSwNgUW5uruzYsaPwti5Up00ekZGR5hzXkTe64LbOGNaw+amnnjIjcAYMGODRulG1+Nm07dBNpk+fLpMmTTLhU/v27c3banUVSwDevRqrM3PmzJFhw4a5vR4ArtWzZ0/zPVz/SAzAu+k4q6SkJNm+fbv5hVSbQUaMGOHpsgCU09GjR03ApO8s1PFWGjTdfPPNkpycbN5VDMB7LFu2THr16nXG/dq8mZ6ebt5JnJKSYv44fOTIEenevbukpaVJixYtPFIvqia3hssAAAAAAAAAAN/glpnLAAAAAAAAAADfQrgMAAAAAAAAALCMcBkAAAAAAAAAYBnhMgAAAAAAAADAMsJlAAAAAAAAAIBlhMsAAAAAAAAAAMsIlwEAAAAAAAAAlhEuAwAAAAAAAAAsI1wGAAAAAAAAAFhGuAwAAAAAAAAAsIxwGQAAAAAAAAAgVv0fRxPHhJU69RQAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 1600x50 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "TSTensor(samples:8, vars:3, len:5000, device=mps:0, dtype=torch.float32)"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X = np.zeros((10, 3, 5000))\n",
    "y = np.random.randint(0,2,X.shape[0])\n",
    "splits = get_splits(y)\n",
    "dls = get_ts_dls(X, y, splits=splits)\n",
    "xb, yb = dls.train.one_batch()\n",
    "xb"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "If you try to use SequencerPlus, it's likely you'll get an 'out-of-memory' error.\n",
    "\n",
    "To avoid this you can subsample the sequence reducing the input's length. This can be done in multiple ways. Here are a few examples: "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([8, 3, 99])"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Separable convolution (to avoid mixing channels)\n",
    "feature_extractor = Conv1d(xb.shape[1], xb.shape[1], ks=100, stride=50, padding=0, groups=xb.shape[1]).to(default_device())\n",
    "feature_extractor.to(xb.device)(xb).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Convolution (if you want to mix channels or change number of channels)\n",
    "feature_extractor=MultiConv1d(xb.shape[1], 64, kss=[1,3,5,7,9], keep_original=True).to(default_device())\n",
    "test_eq(feature_extractor.to(xb.device)(xb).shape, (xb.shape[0], 64, xb.shape[-1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([8, 3, 100])"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# MaxPool\n",
    "feature_extractor = nn.Sequential(Pad1d((0, 50), 0), nn.MaxPool1d(kernel_size=100, stride=50)).to(default_device())\n",
    "feature_extractor.to(xb.device)(xb).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([8, 3, 100])"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# AvgPool\n",
    "feature_extractor = nn.Sequential(Pad1d((0, 50), 0), nn.AvgPool1d(kernel_size=100, stride=50)).to(default_device())\n",
    "feature_extractor.to(xb.device)(xb).shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Once you decide what type of transform you want to apply, you just need to pass the layer as the feature_extractor attribute:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bs = 16\n",
    "nvars = 4\n",
    "seq_len = 1000\n",
    "c_out = 2\n",
    "d_model = 128\n",
    "\n",
    "xb = torch.rand(bs, nvars, seq_len)\n",
    "feature_extractor = partial(Conv1d, ks=5, stride=3, padding=0, groups=xb.shape[1])\n",
    "model = TSSequencerPlus(nvars, c_out, seq_len, d_model=d_model, feature_extractor=feature_extractor)\n",
    "test_eq(model.to(xb.device)(xb).shape, (bs, c_out))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Categorical variables"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tsai.utils import alphabet, ALPHABET"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = alphabet[np.random.randint(0,3,40)]\n",
    "b = ALPHABET[np.random.randint(6,10,40)]\n",
    "c = np.random.rand(40).reshape(4,1,10)\n",
    "map_a = {k:v for v,k in enumerate(np.unique(a))}\n",
    "map_b = {k:v for v,k in enumerate(np.unique(b))}\n",
    "n_cat_embeds = [len(m.keys()) for m in [map_a, map_b]]\n",
    "szs = [emb_sz_rule(n) for n in n_cat_embeds]\n",
    "a = np.asarray(a.map(map_a)).reshape(4,1,10)\n",
    "b = np.asarray(b.map(map_b)).reshape(4,1,10)\n",
    "inp = torch.from_numpy(np.concatenate((c,a,b), 1)).float()\n",
    "feature_extractor = partial(Conv1d, ks=3, padding='same')\n",
    "model = TSSequencerPlus(3, 2, 10, d_model=64, cat_pos=[1,2], feature_extractor=feature_extractor)\n",
    "test_eq(model(inp).shape, (4,2))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Sequence Embedding"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Sometimes you have a samples with a very long sequence length. In those cases you may want to reduce it's length before passing it to the transformer. To do that you may just pass a token_size like in this example:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([8, 128, 168])"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t = torch.rand(8, 2, 10080)\n",
    "SeqTokenizer(2, 128, 60)(t).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([8, 5])"
      ]
     },
     "execution_count": null,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "t = torch.rand(8, 2, 10080)\n",
    "model = TSSequencerPlus(2, 5, 10080, d_model=64, token_size=60)\n",
    "model(t).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/javascript": "IPython.notebook.save_checkpoint();",
      "text/plain": [
       "<IPython.core.display.Javascript object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/Users/nacho/notebooks/tsai/nbs/069_models.TSSequencerPlus.ipynb saved at 2025-03-01 15:24:12\n",
      "Correct notebook to script conversion! 😃\n",
      "Saturday 01/03/25 15:24:15 CET\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "                <audio  controls=\"controls\" autoplay=\"autoplay\">\n",
       "                    <source src=\"data:audio/wav;base64,UklGRvQHAABXQVZFZm10IBAAAAABAAEAECcAACBOAAACABAAZGF0YdAHAAAAAPF/iPh/gOoOon6w6ayCoR2ZeyfbjobxK+F2Hs0XjKc5i3DGvzaTlEaraE+zz5uLUl9f46fHpWJdxVSrnfmw8mYEScqUP70cb0Q8X41uysJ1si6Eh1jYzXp9IE2DzOYsftYRyoCY9dJ/8QICgIcEun8D9PmAaBPlfT7lq4MFIlh61tYPiCswIHX+yBaOqT1QbuW7qpVQSv9lu6+xnvRVSlyopAypbGBTUdSalrSTaUBFYpInwUpxOzhti5TOdndyKhCGrdwAfBUcXIJB69p+Vw1egB76+n9q/h6ADglbf4LvnIHfF/981ODThF4m8HiS0riJVjQ6c+/EOZCYQfJrGrhBmPVNMmNArLKhQlkXWYqhbaxXY8ZNHphLuBJsZUEckCTFVHMgNKGJytIDeSUmw4QN4Qx9pReTgb3vYX/TCBuApf75f+P5Y4CRDdN+B+tngk8c8nt03CKGqipgd13OhotwOC5x9MCAknFFcmlmtPmagFFFYOCo0qRzXMhVi57pryNmIEqJlRi8bm52PfuNM8k4dfQv+4cO12l6zCGdg3jl730uE/KAPvS+f0wEAoAsA89/XfXQgBESIn6S5luDtiC8eh/YmIfpLqt1OMp5jXg8/24MveqUNUnPZsqw0Z3yVDldnaUOqIZfXlKrm36zzWhjRhaT+r+ncHI5/otUzfd2uSt7hl/bqXtoHaCC6+mqfrAOeoDD+PJ/xf8RgLMHfH/b8GeBihZIfSXidoQSJWB52NM1iRkzz3MkxpKPbUCrbDu5d5fgTAxkSK3JoEhYD1p2omere2LZTuqYLbdWa49Cx5Dww7tyXDUnioXRkHhwJyKFvd/AfPoYy4Fl7j1/LQorgEr9/X89+0qAOAwAf13sJoL8Gkd8wt25hWIp3Heez/eKODfPcSPCzpFNRDVqf7UlmnNQKGHgqd+jgVvJVm2f265QZTpLS5byur1tpT6ajvrHq3Q2MXWIxtUCehoj8YMk5LB9hRQegeTypn+nBQWA0QHgf7f2q4C5EFt+5ucOg2YfHXtq2SSHpS0ydnTL4IxFO6pvNb4ulBdInWfcsfSc7VMmXpSmE6eeXmZThJxpsgRohEfOk86+AHCoOpOMFsx1dv8s6oYT2k17uR7ngpXod34IEJqAaPfnfyABCIBZBpl/NPI2gTQVjX134x2ExSPMeR7VtYjZMWJ0W8ftjkA/YW1durCWykvjZFKu4p9LVwVbZKNkqpxh6U+6mRC2mGq2Q3SRvsIgcpc2sIpD0Bp4uiiFhW3ecXxOGgaCDe0Vf4cLPoDv+/5/mfw1gN4KKX+17emBqBmYfBHfVYUZKFR44NBtiv41bHJUwx+RJkP1apu2VJlkTwli4qrwoo1ax1dToNCtemRSTBGXz7kJbdM/PY/Dxht0dTLziH7Ul3loJEiE0uJsfdsVTYGL8Yt/AgcMgHYA7X8S+IqAYA+QfjzpxIIVHnp7tdqzhmAstXaxzEqMETpScGC/dJP3Rmdo8LIZnOVSEF+Opxumsl1sVF+dVrE5Z6NIiZSkvVdv2zsqjdnK8HVDLlyHyNjuegogM4NA5z9+YRG9gA722H97AgOA/gSyf43zCIHdE899yuTIg3ciNXpm1jmImTDwdJPITI4RPhRugbvslbFKt2Vfr/6eTFb4W1WkY6m6YPdQjJr2tNZp3EQlko7BgXHRNz2LAc+gdwMq7IUf3R58ohtFgrbr6n7hDFWAlPr8f/T9I4CECU9/De+vgVQY5nxh4POEzybJeCTS5YnCNAZzhsRzkP1Bsmu4t4aYU07nYuerA6KWWcJYO6HHrKJjaE3Zl624UWz/QOOPjcWHc7QzdIk40yl5tCWjhIDhJX0xF4CBMvBsf10IF4Ac//Z/bPlsgAcOwn6S6n6CwxzUewLcRoYaKzV38M23i9o493CNwL6S1UUuaQe0QpvbUfdfiqglpcRccFU+nkWwambASUiVfLyqbg49xY2eyWh1hy/Sh37XjHpaIYKD7OUEfrgS5IC09MV/1gMBgKMDyH/n9N6AhhINfh7mdoMoIZt6r9fAh1cvfHXNya6N4DzDbqi8K5WWSYlmbbAdnkpV6FxJpWSo1V8DUmGb3rMRaQBG2JJgwN9wCDnNi8HNI3dKK1aG0dvHe/UciIJf6rt+Og5wgDn59X9P/xWAKQhxf2XweYH+FjB9suGVhIMlOnlo02GJhTOdc7vFyo/TQGxs2Li7lz9NwmPurBihnVi7WSWiwKvGYntOpJiOt5drKUKMkFnE8HLxNPmJ9NG4eP8mAYUv4Np8hhi3gdruSX+3CSWAwP38f8f6UoCuDPF+6Os8gnAbKnxQ3d2F0imydzDPKIuiN5lxu8EKkrFE82kftW2az1DbYImpMqTUW3FWIJ83r5hl2koJlla7+m0+PmSOZcjcdMgwS4g11iZ6qCLUg5jkxn0QFA6BWvOvfzEFBIBHAtp/Qfa3gC4RSH5y5yeD2B/8evnYS4cULgR2CMsUja47cG/QvW6UeEhXZ3+xP51GVNVdP6Zpp+1eDFM5nMeySWghR4+TNL85cD46YIyCzKJ2kCzEhoTabXtGHs+CCemJfpMPjoDe9+t/qQALgM8Gj3++8UaBqRV2fQTjO4Q3JKd5r9TgiEYyMHTxxiWPpz8jbfq585YpTJpk960xoKFXsVoTo7yq6GGMTw==\" type=\"audio/wav\" />\n",
       "                    Your browser does not support the audio element.\n",
       "                </audio>\n",
       "              "
      ],
      "text/plain": [
       "<IPython.lib.display.Audio object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#|eval: false\n",
    "#|hide\n",
    "from tsai.export import get_nb_name; nb_name = get_nb_name(locals())\n",
    "from tsai.imports import create_scripts; create_scripts(nb_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "python3",
   "language": "python",
   "name": "python3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
